'use strict'

const initDb = require('./utils/initDb');

initDb();

const cnblogs = require('./models/cnblog');

var Crawler = require("crawler");

var i = 1;
    var c = new Crawler({
        rateLimit: 10,
        maxConnections: 1,
        // This will be called for each crawled page
        callback: function (error, res, done) {
            if (error) {
                console.log(error);
            } else {
                var $ = res.$;
                var post = $(" .dw_table div[class = 'el']").toArray();
                post.forEach(element=>{
                    var staffName = $(element).find(" .t1 ").text().trim().replace(/[ \n\r]/g, ' ');
                    var companyName = $(element).find(" .t2 ").text();
                    var workingPlace = $(element).find(" .t3").text();
                    var salary = $(element).find(" .t4").text();
                    var releaseTime = $(element).find(" .t5").text();
                    cnblogs.create({StaffName:staffName,CompanyName:companyName,WorkingPlace:workingPlace,Salary:salary,ReleaseTime:releaseTime});
                    //console.log(staffName,companyName);
                })
                
                
                // $ is Cheerio by default
                //a lean implementation of core jQuery designed specifically for the server
                
            }
            done();
        }
    });

    // Queue just one URL, with default callback
    //c.queue('https://search.51job.com/list/040000,000000,0000,01,9,99,%2520,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=');
    while(i<=200){
        c.queue('https://search.51job.com/list/040000,000000,0000,01,9,99,%2520,2,' + i +'.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=');
        i=i+1;
    }

